# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import hashlib
import os

import MySQLdb as db
import redis
from scrapy.exceptions import DropItem

from zuwome.items import UserItem, UidItem


#  =====================================================================
#  通过特定的url爬取分析获得uid然后拼接新的路径存入redis数据库
#  进行uid的唯一性校验, 所有的uid存入文件中
#  拼接成新的url存入redis数据库
#  =====================================================================
def genid(item):
    if not isinstance(item, UidItem):
        return None
    text = '%s' % (item.get('uid'))
    h = hashlib.sha1(text.encode())
    return h.hexdigest()

def getUids():
    uids = []
    with open("userData/uids.txt", "r") as f:
        for uid_line in f.readlines():
            uids.append(uid_line.split()[1])
    return uids

class UniquePipeline:

    def process_item(self, item, spider):
        if not isinstance(item, UidItem):
            return None
        item_id = genid(item)
        if item_id in self.all_ids:
            raise DropItem('item exists')
        else:
            self.all_ids.append(item_id)
            return item

    def open_spider(self, spider):
        self.all_ids = []


class Uids2TextPipeline:

    def open_spider(self, spider):
        self.all_ids = []
        self.file = open('userData/uids.txt', 'a+')

    def close_spider(self, spider):
        self.file.close()

    def process_item(self, item, spider):
        if not isinstance(item, UidItem):
            return None
        oldUids = getUids()
        item_id = genid(item)
        if item_id in self.all_ids or item['uid'] in oldUids:
            raise DropItem("uid exists")
        else:
            self.all_ids.append(item_id)
            self.file.write('%s: %s\n' % ('uid', item['uid']))
            return item

class RedisPipeline:

    def __init__(self):
        self.r = redis.Redis(host="localhost", port=6379, db=0)

    def process_item(self, item, spider):
        if not isinstance(item, UidItem):
            return None
        url = "https://v2.zuwome.com/user/%s/detail/?v=2.7.2&dev=android&dev version=6.0&dev name=HUAWEIGRA-UL000&uuid=56af31f4b502da60"
        user_url = url % item['uid']
        self.r.lpush("user_url", user_url)
        return item


#  =====================================================================
#  获取redis数据库中的user_url爬取个人信息,进行完整性校验和单独存入文件中
#  =====================================================================
class IntegrityPipeline:

    def process_item(self, item, spider):
        if not isinstance(item, UserItem):
            return None
        for field in item.fields:
            if field not in item or item[field] is None:
                raise DropItem('invalid item')
        return item

class Peronel2FilePipeline:

    def process_item(self, item, spider):
        if not isinstance(item, UserItem):
            return None
        userinfo_path = "userData"
        if userinfo_path not in os.listdir():
            os.makedirs(userinfo_path)

        fmt = 'uid: %(uid)s\nuser: %(user)s\n'
        content = fmt % item
        fname = userinfo_path + "/" + item.get('uid') + '.txt'
        with open(fname, 'w') as f:
            f.write(content)

        return item
